{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot import Collection\n",
    "from cot.stats import evaluation_as_table"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## filtering datasets"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First selecting the generated cots, then filter samples for samples with generated cots left."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = Collection.load_thoughtsource_100()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll.select_generated_cots(\n",
    "    author=\n",
    "        'thoughtsource',\n",
    "    model=[\n",
    "        # 'gpt-3.5-turbo',\n",
    "        'gpt-4',\n",
    "        # 'flan-T5-xxl',\n",
    "    ],\n",
    "    cot_trigger=[\n",
    "        None, \n",
    "        # 'zhou-01'\n",
    "    ],\n",
    "    instruction=[\n",
    "        None\n",
    "    ]\n",
    "    # answer= False\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 25      | -      |\n",
       "| med_qa         | -       | -       | 27     |\n",
       "| medmc_qa       | -       | 31      | -      |\n",
       "| open_book_qa   | -       | -       | 8      |\n",
       "| strategy_qa    | 29      | -       | -      |\n",
       "| worldtree      | -       | -       | 1      |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll.select_generated_cots(answer=False)\n",
    "coll = coll.filter(lambda x: len(x[\"generated_cot\"])==1)\n",
    "coll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 75      | -      |\n",
       "| med_qa         | -       | -       | 73     |\n",
       "| medmc_qa       | -       | 69      | -      |\n",
       "| open_book_qa   | -       | -       | 92     |\n",
       "| strategy_qa    | 71      | -       | -      |\n",
       "| worldtree      | -       | -       | 99     |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll.select_generated_cots(answer=True)\n",
    "coll = coll.filter(lambda x: len(x[\"generated_cot\"])==1)\n",
    "coll"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
